RAG Evaluation

# pip install llama-index-llms-openai llama-index-embeddings-openai llama-index
from llama_index.evaluation import RetrieverEvaluator
from llama_index.evaluation.retrieval.metrics import RetrievalMetric
from llama_index.llms import OpenAI
from llama_index.core import SimpleDirectoryReader, VectorStoreIndex
import os

# Set your API Key (required for LLM-based query generation and embeddings)
# os.environ["OPENAI_API_KEY"] = "sk-..." 

# --- Setup: Index and Retriever ---
# 1. Load data and create a simple index (assuming local files are present)
documents = SimpleDirectoryReader(input_files=["Assets/paul_graham_essay.txt"]).load_data()
index = VectorStoreIndex.from_documents(documents)
retriever = index.as_retriever(similarity_top_k=5)

# --- Metric Calculation (Simplified) ---

# 2. Define the metrics we want to calculate
# LlamaIndex Metrics are classes, not functions
metrics = [
    RetrievalMetric.HIT_RATE,  # Measures if a relevant document is in top k (Recall-like)
    RetrievalMetric.MRR,       # Mean Reciprocal Rank
    RetrievalMetric.PRECISION, # Precision@k
    RetrievalMetric.NDCG,      # NDCG@k
    RetrievalMetric.AVERAGE_RECALL, # Average Recall
]

# 3. Create the Evaluator
# NOTE: This Evaluator requires a dataset containing queries and ground_truth_nodes
# For simplicity, we assume 'test_dataset' is loaded with these fields.
evaluator = RetrieverEvaluator(
    metrics=metrics,
    retriever=retriever
)

# 4. Run the Evaluation (Requires a test dataset)
# evaluation_results = await evaluator.evaluate_dataset(test_dataset)
# print(evaluation_results.average_scores) 

# pip install llama-index-llms-openai llama-index-core
from llama_index.llms import OpenAI
from llama_index.evaluation import ResponseEvaluator
from llama_index.evaluation.base import EvaluationResult
from typing import List

# 1. Define the LLM-as-Judge model 
# This is the "impartial judge" that determines the scores.
judge_llm = OpenAI(model="gpt-4o") 

# 2. Create the Response Evaluator instance
# The evaluator is set up to use the judge_llm for scoring.
evaluator = ResponseEvaluator(llm=judge_llm)

# --- Metric Definitions ---

def evaluate_response_quality(
        query: str,
    response: str, 
    source_nodes: List[str] # List of text chunks retrieved
) -> EvaluationResult:
    """
    Evaluates response faithfulness and relevancy using LlamaIndex LLM-as-Judge.
    
    Args:
        query: The original user question.
        response: The LLM's final synthesized answer.
        source_nodes: The retrieved contexts used for generation.
        
    Returns: EvaluationResult object containing scores and reasoning.
    """
    
    # 3. Define the metrics and run the evaluation
    # LlamaIndex's ResponseEvaluator requires passing both the query and source nodes.
    # It automatically runs the LLM-as-Judge against internal prompts for:
    #   - Faithfulness (Response is supported by Source Nodes)
    #   - Relevancy (Response directly answers the Query)
    
    evaluation_result = evaluator.evaluate(
        query=query, 
        response=response, 
        contexts=source_nodes # Corresponds to the retrieved 'context' from the trace
    )
    
    return evaluation_result

# --- Example Usage ---

# Simulate data from a production trace
sample_query = "What is the capital of Texas, and where is the population data stored?"
sample_response = "The capital of Texas is Austin. The population data is stored in the PostgreSQL database."
sample_nodes = [
    "Austin is the capital of Texas.",
    "Population figures are maintained in the PostgreSQL DB schema 'public.census_data'."
]

# Run the evaluation
results = evaluate_response_quality(
    query=sample_query,
    response=sample_response,
    source_nodes=sample_nodes
)

print(f"\nResponse Faithfulness Score: {results.passing}: {results.feedback}")
print(f"Full Evaluation Report: {results.json()}")

# 1. Collect real user queries
real_queries = [
    "How do I reset my password?",
    "What's the return policy?",
    "Do you ship internationally?",
    # ... 50-100 real queries
]

# 2. For each query, manually identify relevant docs
golden_dataset = []
for query in real_queries:
    # Human annotators review and mark relevant docs (text content is needed here)
    relevant_node_texts = annotate_relevant_docs_texts(query, corpus) # Must return node TEXT
    
    # Optionally: write expected answer (still useful for the ResponseEvaluator ground truth)
    expected_answer = write_expected_answer(query, relevant_node_texts)
    
    # LlamaIndex RetrieverEvaluator requires this exact format:
    golden_dataset.append({
        "query": query,
        "ground_truth_nodes": relevant_node_texts, # List of Node Texts (the relevant chunks)
        "expected_answer": expected_answer          
    })

# 3. Save for repeated evaluation
import json
with open('golden_dataset.json', 'w') as f:
    json.dump(golden_dataset, f, indent=2)

# Now you can evaluate ANY RAG system against this benchmark

def debug_rag_failure_with_scores(
    query: str, 
    retrieval_scores: dict, # Output from RetrieverEvaluator.average_scores (e.g., {'hit_rate': 0.5, 'mrr': 0.7})
    response_results: EvaluationResult # Output from ResponseEvaluator
):
    """
    Systematic debugging of RAG failures using LlamaIndex metric scores.
    Isolates: retrieval problem vs. generation problem.
    """
    
    print("=== RAG Failure Analysis ===")
    print(f"Query: {query}\n")
    
    # 1. Check Retrieval Failure (Layer 1)
    # We check if a key retrieval metric (Hit Rate or MRR) falls below the target.
    hit_rate = retrieval_scores.get(RetrievalMetric.HIT_RATE, 0)
    mrr_score = retrieval_scores.get(RetrievalMetric.MRR, 0)

    # Check against targets (Recall@10 > 0.90, MRR > 0.80)
    if hit_rate < 0.80 or mrr_score < 0.70:
        print(f"\n❌ RETRIEVAL PROBLEM (Hit Rate: {hit_rate:.2f}, MRR: {mrr_score:.2f})")
        print("— Fix: Focus on tuning chunking, reranking, or embedding model.")
        return "retrieval_failure"
    
    # 2. Check Generation Failure (Layer 2)
    # Check Faithfulness and Relevancy directly from the ResponseEvaluator output.
    
    # NOTE: LlamaIndex returns True/False for 'passing' and the reason in 'feedback'
    if not response_results.passing:
        print("\n❌ GENERATION PROBLEM (LLM-as-Judge Failure)")
        
        # We need to parse the feedback to diagnose Faithfulness vs. Relevancy failure
        feedback = response_results.feedback
        
        if "Faithfulness" in feedback or "support" in feedback:
            print(f"— Failure Type: Hallucination/Misinterpretation (Faithfulness)")
            print(f"— Feedback: {feedback}")
            print("— Fix: Constrain LLM prompt, adjust temperature, or improve reranker quality to reduce noise.")
            return "generation_faithfulness_failure"
    
        if "Relevance" in feedback or "query" in feedback:
            print(f"— Failure Type: Off-Topic/Irrelevance (Relevancy)")
            print(f"— Feedback: {feedback}")
            print("— Fix: Clarify LLM system prompt on staying concise and relevant.")
            return "generation_relevancy_failure"
    
    # If both layers pass
    print("\n✓ Both retrieval and generation meet automated targets.")
    return "evaluation_passed"

import random
import numpy as np
from llama_index.evaluation import RetrieverEvaluator, ResponseEvaluator

# Assume setup from previous sections:
# - evaluator_retrieval (RetrieverEvaluator instance with metrics configured)
# - evaluator_response (ResponseEvaluator instance with judge_llm configured)
# - rag_system (your end-to-end RAG pipeline)

class ProductionRAGMonitor:
    """Continuous evaluation using LlamaIndex modules in production."""
    
    def __init__(self, golden_dataset: list, sample_rate: float = 0.1):
        # The golden_dataset now contains {'query', 'ground_truth_nodes', 'expected_answer'}
        self.golden_dataset = golden_dataset
        self.sample_rate = sample_rate
        # Evaluators are configured outside this class and passed in (or initialized)
        self.ret_evaluator = RetrieverEvaluator(retriever=rag_system.retriever, metrics=metrics)
        self.res_evaluator = ResponseEvaluator(llm=judge_llm)
    
    def should_evaluate(self) -> bool:
        """Sample queries for evaluation (to manage cost)."""
        return random.random() < self.sample_rate
    
    def log_and_evaluate(
        self,
        query: str,
        retrieved_nodes_text: list[str], # Text list of the nodes returned by the RAG system
        answer: str
    ):
        """Log every query, evaluate sample against a golden case if found."""
        log_query(query, retrieved_nodes_text, answer)
        
        if self.should_evaluate():
            # Find matching golden case (requires exact query match for simplicity here)
            golden_case = next(
                (c for c in self.golden_dataset if c['query'] == query),
                None
            )
            
            if golden_case:
                # --- Step 1: Evaluate Retrieval (Layer 1) ---
                # This requires constructing a mini-dataset for the RetrieverEvaluator
                # In a real system, you'd evaluate the original retriever component.
                
                # --- Step 2: Evaluate Generation (Layer 2) ---
                res_eval = self.res_evaluator.evaluate(
                    query=query, 
                    response=answer, 
                    contexts=retrieved_nodes_text 
                )
                
                # Check critical metrics (Faithfulness and Relevance)
                # NOTE: ResponseEvaluator's result.passing often checks Faithfulness
                if not res_eval.passing:
                    alert(f"Hallucination/Irrelevance Detected: {res_eval.feedback}")
    
    def weekly_report(self):
        """Aggregate metrics from week's queries and flag degradation."""
        # Analyze logged queries, check averages of NDCG, MRR, Faithfulness, etc. 
        # against historical baseline metrics.
        pass

# Integration in the RAG endpoint remains the same, ensuring the monitor is called 
# with the output of the unified LlamaIndex RAG system.

# Provided: starter/lesson_2.6/rag_system.py (working RAG)
# Provided: starter/lesson_2.6/test_cases.json (20 test cases)

# Your task:
# 0. Corpus: use chunks from 'Assets/paul_graham_essay.txt' for retrieval
# 1. Load test cases (or create 20 Q/A pairs about the essay if missing)
# 2. Run RAG system on all queries
# 3. Use LlamaIndex evaluations to compute retrieval metrics (Recall@5, Precision@5, NDCG@10)
# 4. Use LlamaIndex evaluations for generation (Faithfulness, Relevance)
# 5. Identify 3 worst-performing queries
# 6. Debug: Is failure in retrieval or generation?

# Expected insights:
# - Retrieval usually bottleneck (70% of failures)
# - Multi-hop queries hardest
# - Ambiguous queries need query refinement